package Image;


import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;


/*
 * IMPORTANT: Make sure that you update crawler4j.properties file and set
 * crawler.include_images to true
 */

public class ImageCrawlController {

    public static void main(String[] args) throws Exception {

        String rootFolder = "C:\\Users\\User\\Documents\\Eclipse_Projects\\Maven_Testing_SVN\\src\\main\\java";//args[0];
        int numberOfCrawlers = 10;//Integer.parseInt(args[1]);
        String storageFolder = "bla";//args[2];

        CrawlConfig config = new CrawlConfig();

        config.setCrawlStorageFolder(rootFolder);
        //config.setResumableCrawling(true);
        /*
         * Since images are binary content, we need to set this parameter to
		 * true to make sure they are included in the crawl.
		 */
        config.setIncludeBinaryContentInCrawling(false);

        //config.setUserAgentString("Nanozilla/5.0 (WinDoge NT 6.1; WOW SUCH FACTOR; DOGEBROWSER:WOOOW; rv:1.0) like NOT");
      
        //config.setUserAgentString("Iasda");
        //config.setProxyHost("68.186.172.52");
        //config.setProxyPort(28153);

        config.setUserAgentString("Morilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko");
        String s = "http://www.fct.unl.pt/";

        //PolitenessDelay de 2 segundos
        config.setPolitenessDelay(200);


        PageFetcher pageFetcher = new PageFetcher(config);
        RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
        RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
        CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);
        controller.setCustomData(s);
        controller.addSeed(s);

        System.out.println("A lan�ar o crawl");
        //controller.startNonBlocking(ImageCrawler.class, numberOfCrawlers);
        controller.startNonBlocking(ImageCrawler.class, numberOfCrawlers);
        Thread.sleep(3*60*1000);
        // Wait for 10 s
        //Thread.sleep(5 * 1000);
        //System.out.println("Asked for a shutdown for all 'people'");
        // Send the shutdown request and then wait for finishing
        controller.shutdown();
        ImageCrawler.fop.flush();
        ImageCrawler.fop.close();
        controller.waitUntilFinish();
        System.out.println("Finished");
    }

}
